'''
rcs.py

Copyright 2012 Tomas Velazquez

This file is part of w3af, w3af.sourceforge.net .

w3af is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation version 2 of the License.

w3af is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with w3af; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

'''
import re

import core.controllers.outputManager as om
import core.data.kb.knowledgeBase as kb
import core.data.kb.vuln as vuln
import core.data.constants.severity as severity

from core.controllers.plugins.crawl_plugin import CrawlPlugin
from core.controllers.w3afException import w3afException
from core.data.bloomfilter.bloomfilter import scalable_bloomfilter

from core.controllers.core_helpers.fingerprint_404 import is_404
import struct


class rcs(CrawlPlugin):
    '''
    Search Git, Mercurial (HG), Bazaar (BZR), Subversion (SVN) and CVS 
    repositories and checks for files containing

    This plugin rewrite find_dvcs.py from Adam Baldwin (adam_baldwin@ngenuity-is.com)
    
    @author: Tomas Velazquez ( tomas.velazquezz@gmail.com )
    '''

    def __init__(self):
        CrawlPlugin.__init__(self)
        
        # Internal variables
        self._analyzed_dirs = scalable_bloomfilter()
        

        self._rcs = [{'name':'git repository','filename':'.git/index','function':self.git_index},
              {'name':'git ignore','filename':'.gitignore','function':self.ignore_file},
              {'name':'hg repository','filename':'.hg/store/fncache','function':self.hg_store_fncache},
              {'name':'hg ignore','filename':'.hgignore','function':self.ignore_file},
              {'name':'bzr repository','filename':'.bzr/checkout/dirstate','function':self.bzr_checkout_dirstate},
              {'name':'bzr ignore','filename':'.bzrignore','function':self.ignore_file},
              {'name':'svn repository','filename':'.svn/entries','function':self.svn_entries},
              {'name':'svn ignore','filename':'.svnignore','function':self.ignore_file},
              {'name':'cvs repository','filename':'CVS/Entries','function':self.cvs_entries},
              {'name':'cvs ignore','filename':'.cvsignore','function':self.ignore_file}]


    def crawl(self, fuzzable_request ):
        '''
        For every directory, fetch a list of files and analyze the response
        using regex.
        
        @parameter fuzzable_request: A fuzzable_request instance that contains
                                    (among other things) the URL to test.
        '''
        domain_path = fuzzable_request.getURL().getDomainPath()
        
        if domain_path not in self._analyzed_dirs:
            self._analyzed_dirs.add( domain_path )

            test_generator = self._url_generator( domain_path )
            
            self._tm.threadpool.map_multi_args(self._send_and_check, test_generator)
    
    def _get_all_directories( self, filenames ):
            resources = set()
            _filenames = filter(lambda x:x not in self._analyzed_dirs, filenames)

            for line in _filenames:
                if line.startswith('/'):
                    line = line[1:]
                if line.startswith('./'):
                    line = line[2:]
                if line.endswith('/'):
                    line = line[:-1]

                if not '/' in line:
                    resources.add(line)
                else:
                    directory = line.split('/')
                    for i in range(1, len(directory) + 1):
                        resources.add( '/'.join(directory[0:i]) )

            return resources

    def _send_and_check(self, repo_url, function, repo, domain_path, relative_url):
        try:
            response = self._uri_opener.GET( repo_url, cache=True )
        except w3afException:
            om.out.debug('Failed to GET '+repo+' file: "' + repo_url + '"')
        else:
            if not is_404(response):
                filenames = function(response.getBody())
                filenames.add(relative_url)
                filenames = self._get_all_directories(filenames)

                for filename in filenames:
                    try:
                        repo_url = domain_path.urlJoin(filename)
                        resp = self._uri_opener.GET( repo_url, cache=True )
                    except w3afException:
                        om.out.debug('Failed to GET '+repo+' file: "' + repo_url + '"')

                    if not is_404(resp):
                        for fr in self._create_fuzzable_requests( resp ):
                            self.output_queue.put(fr)
                    self._analyzed_dirs.add(filename)

                v = vuln.vuln()
                v.setPluginName(self.getName())
                v.set_id( response.id )
                v.setName( 'Possible '+repo+' found' )
                v.setSeverity(severity.LOW)
                v.setURL( response.getURL() )
                msg = ('A %s file was found at: "%s"; this could'
                       ' indicate that a %s is accessible. You might'
                       ' be able to download the Web application source code.')
                v.setDesc( msg % (repo, v.getURL(), repo) )
                kb.kb.append( self, repo.upper(), v )
                om.out.vulnerability( v.getDesc(), severity=v.getSeverity() )

    def _url_generator(self, domain_path):
        for repo in self._rcs:
            relative_url = repo['filename']
            function = repo['function']
            repo_url = domain_path.urlJoin(relative_url)
            repo = repo['name']
            yield repo_url, function, repo, domain_path, relative_url

    def git_index( self, f ):
        filenames = set()
        signature = 'DIRC'
        offset = 12

        if f[:4] != signature:
            return

        version, = struct.unpack('>I', f[4:8])
        index_entries, = struct.unpack('>I', f[8:12])

        if version == 2: filename_offset = 62
        elif version == 3: filename_offset = 63
        else: return

        for i in range(0, index_entries):
            offset += filename_offset - 1
            length, = struct.unpack('>B', f[offset:offset+1])
            filename = f[offset+1:offset+1+length]
            padding = 8 - ((filename_offset + length) % 8)
            om.out.debug(filename)
            filenames.add(filename)
            offset += length + 1 + padding

        return filenames

    def hg_store_fncache( self, f ):
        filenames = set()
        for line in f.split('\n'):
            om.out.debug('.hg/'+line)
            filenames.add('.hg/'+line)
            if line.startswith('data/') and line.endswith('.i'):
                om.out.debug(line[5:-2])
                filenames.add(line[5:-2])

        return filenames

    def bzr_checkout_dirstate( self, f ):
        filenames = set()
        header = '#bazaar dirstate flat format '

        if f[0:29] != header:
            return

        f = f.split('\x00')
        found = True
        for offset in range(0,len(f)):
            filename = f[offset-2]
            if f[offset] == 'd':
                if found:
                    om.out.debug(filename)
                    filenames.add(filename)
                found = not found
            if f[offset] == 'f':
                if found:
                    om.out.debug(filename)
                    filenames.add(filename)
                found = not found

        return filenames

    def svn_entries( self, f ):
        filenames = set()
        lines = f.split('\n')
        offset = 29

        while offset < len(lines):
            line = lines[offset].strip()
            filename = lines[offset-1].strip()
            if line == 'file':
                om.out.debug(filename)
                filenames.add(filename)
                offset += 34
            elif line == 'dir':
                om.out.debug(filename)
                filenames.add(filename)
                offset += 3
            else:
                om.out.debug(line+': '+filename)
                break

        return filenames

    def cvs_entries( self, f ):
        filenames = set()
        lines = f.split('\n')

        for line in lines:
            if '/' in line:
                z = line.split('/')
                om.out.debug(z[1])
                filenames.add(z[1])

        return filenames

    def filter_special_character( self, line ):
        special_characters = ['*', '?', '[', ']', ':']
        for char in special_characters:
            if char in line:
                l = line.split(char)[0]
                if '/' in l:
                    line = '/'.join(l.split('/')[:-1])
                else:
                    line = ''
                    break

        return line

    def ignore_file( self, f ):
        filenames = set()
        for line in f.split('\n'):
            if line.startswith('#') or line == '':
                continue

            line = self.filter_special_character(line)
            if not line:
                continue

            if line.startswith('/') or line.startswith('^'):
                line = line[1:]
            if line.endswith('/') or line.endswith('$'):
                line = line[:-1]

            om.out.debug(line)
            filenames.add(line)

        return filenames

    def get_long_desc( self ):
        '''
        @return: A DETAILED description of the plugin functions and features.
        '''
        return '''
    	This plugin search git, hg, bzr, svn or cvs repositories and checks for files containing.

        For example, if the input is:
            - http://host.tld/w3af/index.php
            
        The plugin will perform requests to:
            - http://host.tld/w3af/.git/index
            - http://host.tld/w3af/.gitignore
            - http://host.tld/w3af/.hg/store/fncache
            - http://host.tld/w3af/.hgignore
            - http://host.tld/w3af/.bzr/checkout/dirstate
            - http://host.tld/w3af/.bzrignore
            - http://host.tld/w3af/.svn/entries
            - http://host.tld/w3af/.svnignore
            - http://host.tld/w3af/CVS/Entries
            - http://host.tld/w3af/.cvsignore
            - http://host.tld/.git/index
            - http://host.tld/.gitignore
            - http://host.tld/.hg/store/fncache
            - http://host.tld/.hgignore
            - http://host.tld/.bzr/checkout/dirstate
            - http://host.tld/.bzrignore
            - http://host.tld/.svn/entries
            - http://host.tld/.svnignore
            - http://host.tld/CVS/Entries
            - http://host.tld/.cvsignore
        '''

